{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#from sklearn.preprocessing import LabelEncoder\n",
    "#le = LabelEncoder()\n",
    "#df['sub_grade']= le.fit_transform(df['sub_grade'])\n",
    "#df.head()\n",
    "\n",
    "# home_ownership -> label\n",
    "#df.home_ownership.describe()\n",
    "\n",
    "#le = LabelEncoder()\n",
    "#df['home_ownership']= le.fit_transform(df['home_ownership'])\n",
    "#df.head()\n",
    "\n",
    "# verification_status -> label\n",
    "#df.verification_status.describe()\n",
    "\n",
    "#df.verification_status.unique()\n",
    "\n",
    "#le = LabelEncoder()\n",
    "#df['verification_status']= le.fit_transform(df['verification_status'])\n",
    "#df.head()\n",
    "\n",
    "# purpose -> label\n",
    "#df.purpose.describe()\n",
    "\n",
    "#le = LabelEncoder()\n",
    "#df['purpose']= le.fit_transform(df['purpose'])\n",
    "#df.head()\n",
    "\n",
    "# addr_state -> label\n",
    "#df.addr_state.describe()\n",
    "\n",
    "#le = LabelEncoder()\n",
    "#df['addr_state']= le.fit_transform(df['addr_state'])\n",
    "#df.head()\n",
    "\n",
    "# initial_list_status\n",
    "#df.initial_list_status.describe()\n",
    "\n",
    "#df.initial_list_status.unique()\n",
    "\n",
    "#df.initial_list_status = df.initial_list_status.map({'w':0, 'f': 1})\n",
    "\n",
    "# application_type\n",
    "#df.application_type.describe()\n",
    "\n",
    "#df.application_type.unique()\n",
    "\n",
    "#df.application_type = df.application_type.map({'Individual':0, 'Joint App':1})\n",
    "\n",
    "df.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Xgboost (tree-based)\n",
    "\n",
    "Pro: \n",
    "- Heterogeneous data - Features measured on different scale such as employment length vs annual income\n",
    "- Supports different loss function\n",
    "- Automatically detects (non-linear) feature interactions\n",
    "\n",
    "Con:\n",
    "- requires careful tuning\n",
    "- slow to train \n",
    "- cannot extrapolate. \n",
    "\n",
    "Loss Function:\n",
    "\n",
    "Squared loss minimizes expectation. Pick it when learning expected return on a stock.\n",
    "\n",
    "Logistic loss minimizes probability. Pick it when learning probability of click on advertisement.\n",
    "\n",
    "Hinge loss minimizes the 0,1 / yes-no question. Pick it when you want a hard prediction.\n",
    "\n",
    "Quantile loss minimizes the median. Pick it when you want to predict house prices.\n",
    "\n",
    "Ensembling multiple loss functions\n",
    "https://github.com/JohnLangford/vowpal_wabbit/wiki/Loss-functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb = XGBClassifier(random_state=0)\n",
    "parameters = {\n",
    "    'learning_rate': [0.025,0.05,0.1], \n",
    "    'max_depth': [3,4,5],\n",
    "    'subsample': [0.25,0.5,1.0],\n",
    "    'reg_lambda': [1,2,3], \n",
    "    'reg_alpha': [0,1,2]\n",
    "}\n",
    "\n",
    "#warnings.filterwarnings('ignore', category=DeprecationWarning) \n",
    "Grid_XGB = GridSearchCV(xgb, parameters, scoring='roc_auc', n_jobs=-1)\n",
    "Grid_XGB.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb = Grid_XGB.best_estimator_\n",
    "print 'Best score: ', Grid_XGB.best_score_\n",
    "print 'Best parameters set: \\n', Grid_XGB.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import ensemble\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "\n",
    "param_grid = {'learning_rate': [0.1, 0.05, 0.02, 0.01],\n",
    "              'max_depth': [4, 6],\n",
    "              'min_samples_leaf': [3, 5, 9, 17],\n",
    "              'max_features': [1.0, 0.3, 0.1]\n",
    "              }\n",
    "# param_grid = {'learning_rate': [0.1],\n",
    "#               'max_depth': [4],\n",
    "#               'min_samples_leaf': [3],\n",
    "#               'max_features': [1.0],\n",
    "#               }\n",
    "\n",
    "grid_gbr = GridSearchCV(ensemble.GradientBoostingRegressor(n_estimators=100),\n",
    "                   param_grid, n_jobs=4, refit=True)\n",
    "\n",
    "grid_gbr.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "est = ensemble.GradientBoostingRegressor(n_estimators=2000).fit(X_train, y_train)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
